import java.io.IOException;
import java.net.URL;
import java.net.URLConnection;
import java.util.HashSet;
import java.util.NoSuchElementException;
import java.util.Scanner;
import java.util.concurrent.ConcurrentLinkedQueue;

// Put the set in the parser not the pageRetriever.
//asf
public class PageRetriever extends Thread
{
	private ConcurrentLinkedQueue<Page>	pageBuffer;
	private ConcurrentLinkedQueue<URL>	pageToRetrieveQueue;
	private DataGatherer gatherer;
	private static final String		END_OF_INPUT = "\\Z";
	private Stopper finished;

	public PageRetriever( ConcurrentLinkedQueue<Page> pageBuffer, ConcurrentLinkedQueue<URL> pageToRetrieveQueue,
						  DataGatherer gatherer, Stopper finished )
	{
		this.pageBuffer = pageBuffer;
		this.pageToRetrieveQueue = pageToRetrieveQueue;
		this.gatherer = gatherer;
		this.finished = finished;
	}

	public void getPageContent( URL url )
	{
		try
		{
			URLConnection connection = url.openConnection();
			Scanner scanner = new Scanner( connection.getInputStream() );
			scanner.useDelimiter( END_OF_INPUT );
			String result = scanner.next();
			pageBuffer.add( new Page( url.toString(), result ) );
		}
		catch ( IOException ex )
		{
			//System.out.println("IO exception: " + url.toString() );
		}
		catch (NoSuchElementException noSuch)
		{
			//System.out.println("bad url: " + url.toString() );
		}
	}
	
	public void run()
	{
		while( gatherer.getPagesRetrieved() < DataGatherer.PAGE_LIMIT )
		{
			if ( !pageToRetrieveQueue.isEmpty() )
			{
				finished.setRetrieverWaiting( false );
				//System.out.println("retrieving");
				
				URL url = pageToRetrieveQueue.poll();
				getPageContent( url );
			}
			else
			{
				if ( pageBuffer.isEmpty() )
				{
					finished.setRetrieverWaiting( true );
					
					// Nothing left to do
					if ( finished.getParserWaiting() && finished.getRetrieverWaiting() )
					{
						break;
					}
				}
				//System.out.println("retriever wait");
				try
				{
					sleep(10);
				}
				catch( InterruptedException e)
				{
					
				}
			}
		}
	}
}
